package com.itdaan.portal.common.util;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.GZIPInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * HTTP连接工具类
 * @author www.itdaan.com
 */
public class HttpUtil {

	static Logger log = LoggerFactory.getLogger(HttpUtil.class);
	
	//每一个web都有唯一的HttpUtil对象 用于间隔多少秒访问一次网站
	private static Map<String,HttpUtil> webHttp = new HashMap<String, HttpUtil>();
	
	public static HttpUtil getInstance(String webId){
		if(!webHttp.containsKey(webId)){
			HttpUtil hu = new HttpUtil();
			webHttp.put(webId, hu);
			return hu;
		}else{
			return webHttp.get(webId);
		}
	}
	
	private HttpUtil(){
		
	}

	public synchronized String getHtmlContent(String url,String encoding) throws Exception {
		Thread.sleep(1000);
		BufferedReader bufferedReader = null;
		StringBuilder html = new StringBuilder();
		URLConnection connection = null;
		HttpURLConnection conn= null;
		InputStream inputStream = null;
		try{
			connection = new URL(url).openConnection();
			conn = (HttpURLConnection)connection;
			connection.setReadTimeout(5*1000);
			int rand = (int) (Math.random() * 6)+1;
			//为防止服务器发现使用各大搜索引擎的爬虫User-Agent，伪装成搜索引擎爬虫。
			if(rand == 1){//谷歌蜘蛛
				connection.setRequestProperty("User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)");
			}else if(rand == 2){//搜狗蜘蛛
				connection.setRequestProperty("User-Agent", "Sogou web spider/3.0(http://www.sogou.com/docs/help/webmasters.htm#07)");
			}else if(rand == 3){//百度蜘蛛
				connection.setRequestProperty("User-Agent", "Baiduspider (http://www.baidu.com/search/spider.htm)");
			}else if(rand == 4){//雅虎蜘蛛
				connection.setRequestProperty("User-Agent", "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)");
			}else if(rand == 5){//SOSO蜘蛛
				connection.setRequestProperty("User-Agent", "Sosospider+(+http://help.soso.com/webspider.htm)");
			}else if(rand == 6){
				connection.setRequestProperty("User-Agent", "Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; )");
			}else{
				connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0");
			}
			connection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
			connection.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
			connection.setRequestProperty("Cookie", "BAIDU_DUP_lcr=http://www.baidu.com/s?wd=%E5%A6%82%E4%BD%95%E4%BC%AA%E9%80%A0cookie&rsv_spt=1&issp=1&f=8&rsv_bp=0&rsv_idx=2&ie=utf-8&tn=monline_5_dg&rsv_enter=1&rsv_sug3=8&rsv_sug4=706&rsv_sug1=7&rsv_pq=de7af6ca000072f2&rsv_t=3ffeojp6IhAdeQBf%2FgBCjFWlZNx7i0GzBGNTIQEVg6vRAukbpQ%2BtAPLJDNgyFotOuiHk&rsv_sug2=0&inputT=9138; __utma=17226283.1486181597.1416578026.1421240038.1421243870.55; __utmz=17226283.1421243870.55.50.utmcsr=baidu|utmccn=(organic)|utmcmd=organic|utmctr=%E5%A6%82%E4%BD%95%E4%BC%AA%E9%80%A0cookie; uuid_tt_dd=2115453778597858656_20141121; __message_gu_msg_id=0; __message_cnel_msg_id=0; bdshare_firstime=1416711446651; __gads=ID=9f762613e4245984:T=1416750519:S=ALNI_Mbkf-DWTDUDyhykdGzqtZayquF62g; __message_in_school=0; UN=jazz2013; UE=\"email@zhangjie.hn.cn\"; __message_sys_msg_id=2910; lzstat_uv=2168163668287810418|3497199; __message_district_code=000000; csdn_has_new_product=2; __cc_verify__aqb_=x/NtDaVdnf6E36sYnlSjQmoz3Nrr+0SP; _csdn_newbbs_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFRkkiJWQ3MDVhYTE3M2NkYTE0ZWQzMmE4NWM4NDVkMjQ4ZDE4BjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMXBrQmQ2cXNaSzJ5OXNoUkZqeUU4UjhzQ2Qvd2JmRlc2dkcvYjJxUXo1NUE9BjsARg%3D%3D--4571a72772ad2a65e1686e830f703cc2dc91bc6e; __utmc=17226283; dc_tos=ni67g3; __utmb=17226283.6.10.1421243870; dc_session_id=1421244723742; __utmt=1");
			
			Map<String, List<String>> header = conn.getHeaderFields();
			Boolean isGzip = false;
			for (Map.Entry<String, List<String>> entry : header.entrySet()) {
			    String key = entry.getKey();
			    for (String value : entry.getValue()) {
			        if ("Content-Encoding".equals(key)&&"gzip".equals(value)) {
			        	isGzip= true;
					}
			    }
			}
			
			if (isGzip==true) { //被压缩过
				inputStream =new GZIPInputStream(connection.getInputStream());
			}else {
				inputStream = connection.getInputStream();
			}
			
			bufferedReader = new BufferedReader(new InputStreamReader(inputStream, encoding));
			int brline = 0;
			String line = "";
			while ((line = bufferedReader.readLine()) != null) {
				//如果含有pre或者code说明是代码区域 需要换行 
				if((line.indexOf("<pre") != -1 && line.indexOf("</pre>") != -1) || (line.indexOf("<code") != -1 && line.indexOf("</code>") != -1)){
					brline = 0;
				}else if(line.indexOf("<pre") != -1 || line.indexOf("<code") != -1){
					brline = 1;
				}else if(line.indexOf("</pre>") != -1 || line.indexOf("</code>") != -1){
					brline = 0;
				}
				if(brline == 0){
					html.append(line);
				}else if(brline == 1){
					if(line.indexOf("<br") == -1){
						html.append(line+"<br/>");
					}else{
						html.append(line);
					}
				}
			}
		}catch(Exception ex){
			throw new Exception("爬取失败：".concat(ex.toString()));
		}finally{
			try {
				if (conn != null) {
					conn.disconnect();
					conn = null;
				}
				if (bufferedReader != null) {
					bufferedReader.close();
					bufferedReader = null;
				}
				if(inputStream != null){
					inputStream.close();
					inputStream = null;
				}
			} catch (IOException e) {
				log.error("{}", e);
			}
		}
		return html.toString();
	}
	
	/**
	 * 直接供获取HTML代码使用
	 */
	public static String getHtml(String url,String encoding){
		BufferedReader bufferedReader = null;
		StringBuilder html = new StringBuilder();
		URLConnection connection = null;
		HttpURLConnection conn= null;
		InputStream inputStream = null;
		try{
			connection = new URL(url).openConnection();
			conn = (HttpURLConnection)connection;
			connection.setReadTimeout(5*1000);
			int rand = (int) (Math.random() * 6)+1;
			//为防止服务器发现使用各大搜索引擎的爬虫User-Agent，伪装成搜索引擎爬虫。
			if(rand == 1){//谷歌蜘蛛
				connection.setRequestProperty("User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)");
			}else if(rand == 2){//搜狗蜘蛛
				connection.setRequestProperty("User-Agent", "Sogou web spider/3.0(http://www.sogou.com/docs/help/webmasters.htm#07)");
			}else if(rand == 3){//百度蜘蛛
				connection.setRequestProperty("User-Agent", "Baiduspider (http://www.baidu.com/search/spider.htm)");
			}else if(rand == 4){//雅虎蜘蛛
				connection.setRequestProperty("User-Agent", "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)");
			}else if(rand == 5){//SOSO蜘蛛
				connection.setRequestProperty("User-Agent", "Sosospider+(+http://help.soso.com/webspider.htm)");
			}else if(rand == 6){
				connection.setRequestProperty("User-Agent", "Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; )");
			}else{
				connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0");
			}
			connection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
			connection.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
			connection.setRequestProperty("Cookie", "BAIDU_DUP_lcr=http://www.baidu.com/s?wd=%E5%A6%82%E4%BD%95%E4%BC%AA%E9%80%A0cookie&rsv_spt=1&issp=1&f=8&rsv_bp=0&rsv_idx=2&ie=utf-8&tn=monline_5_dg&rsv_enter=1&rsv_sug3=8&rsv_sug4=706&rsv_sug1=7&rsv_pq=de7af6ca000072f2&rsv_t=3ffeojp6IhAdeQBf%2FgBCjFWlZNx7i0GzBGNTIQEVg6vRAukbpQ%2BtAPLJDNgyFotOuiHk&rsv_sug2=0&inputT=9138; __utma=17226283.1486181597.1416578026.1421240038.1421243870.55; __utmz=17226283.1421243870.55.50.utmcsr=baidu|utmccn=(organic)|utmcmd=organic|utmctr=%E5%A6%82%E4%BD%95%E4%BC%AA%E9%80%A0cookie; uuid_tt_dd=2115453778597858656_20141121; __message_gu_msg_id=0; __message_cnel_msg_id=0; bdshare_firstime=1416711446651; __gads=ID=9f762613e4245984:T=1416750519:S=ALNI_Mbkf-DWTDUDyhykdGzqtZayquF62g; __message_in_school=0; UN=jazz2013; UE=\"email@zhangjie.hn.cn\"; __message_sys_msg_id=2910; lzstat_uv=2168163668287810418|3497199; __message_district_code=000000; csdn_has_new_product=2; __cc_verify__aqb_=x/NtDaVdnf6E36sYnlSjQmoz3Nrr+0SP; _csdn_newbbs_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFRkkiJWQ3MDVhYTE3M2NkYTE0ZWQzMmE4NWM4NDVkMjQ4ZDE4BjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMXBrQmQ2cXNaSzJ5OXNoUkZqeUU4UjhzQ2Qvd2JmRlc2dkcvYjJxUXo1NUE9BjsARg%3D%3D--4571a72772ad2a65e1686e830f703cc2dc91bc6e; __utmc=17226283; dc_tos=ni67g3; __utmb=17226283.6.10.1421243870; dc_session_id=1421244723742; __utmt=1");
			
			Map<String, List<String>> header = conn.getHeaderFields();
			Boolean isGzip = false;
			for (Map.Entry<String, List<String>> entry : header.entrySet()) {
			    String key = entry.getKey();
			    for (String value : entry.getValue()) {
			        if ("Content-Encoding".equals(key)&&"gzip".equals(value)) {
			        	isGzip= true;
					}
			    }
			}
			
			if (isGzip==true) { //被压缩过
				inputStream =new GZIPInputStream(connection.getInputStream());
			}else {
				inputStream = connection.getInputStream();
			}
			
			bufferedReader = new BufferedReader(new InputStreamReader(inputStream, encoding));
			String line = "";
			while ((line = bufferedReader.readLine()) != null) {
				html.append(line);
			}
		}catch(Exception ex){
			ex.printStackTrace();
			return "";
		}finally{
			try {
				if (conn != null) {
					conn.disconnect();
					conn = null;
				}
				if (bufferedReader != null) {
					bufferedReader.close();
					bufferedReader = null;
				}
				if(inputStream != null){
					inputStream.close();
					inputStream = null;
				}
			} catch (IOException e) {
				log.error("{}", e);
			}
		}
		return html.toString();
	} 	
	
	public static void main(String[] args) {
		String url = "http://www.itdaan.com/post-1-1000.html";
		HttpUtil u = new HttpUtil();
		try {
			String content = u.getHtmlContent(url, "utf-8");
			System.out.println(content);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
}
